import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URLEncoder;
import java.sql.SQLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.Connection.Response;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class AmazonCrawler {

	private int pageCount = 100;
	private static java.sql.Connection connection;
	ExecutorService executor;

	public AmazonCrawler() throws SQLException, IOException {
		
		connection = DB.getConnectionRead();
		connection.setAutoCommit(false);
		File dir = new File("images/");
		if (!dir.exists()) {
				dir.mkdir();
		
		}
		executor = Executors.newFixedThreadPool(2);

	}

	public static void main(String[] args) throws IOException, SQLException, ParseException {
		AmazonCrawler amazonCrawler = new AmazonCrawler();
		System.setProperty("socksProxyHost", "localhost");
		System.setProperty("socksProxyPort", "9050");
		amazonCrawler.run();

		connection.close();
		
		
	}
	public void run() throws SQLException, IOException, ParseException{
  		
		List<SearchAsin> searchAsins = SearchAsin.getSearchAsins(connection);
		for (SearchAsin searchAsin : searchAsins) {
  			System.out.println(new Date()+" - Start crawling data from keyword: "+searchAsin.getAsin_temp());

			crawl(searchAsin);
  			System.out.println(new Date()+" - Finish crawling data from keyword: "+searchAsin.getAsin_temp());

		}
	  		executor.shutdown();
  		System.out.println("images downloading...");

  		while (!executor.isTerminated()) {

		}
		System.out.println("Finished all download threads");
  			
  	}

	public void crawl(SearchAsin searchAsin) throws IOException, SQLException {
		String url = "http://www.amazon.com/s/ref=nb_sb_noss?url=search-alias%3Daps&field-keywords="
				+ URLEncoder.encode(searchAsin.getAsin_temp());

		int max_pages = 10;
		boolean loop = true;
		int page = 1;
		int total = 0;
		while (loop) {
			Document document1 = getConnection(url + "&page=" + page).get();

			Elements div_results = document1.select("div[id^=result_]");
			if (page == 1) {
				String total_str = document1.select("div#resultCount").text();
				try {
					total_str = total_str.split("of")[1];
					total = Integer.parseInt(getNumber(total_str));
					searchAsin.setNumberProducts(total);
					SearchAsin.upDateSearchAsins(connection, searchAsin);
				} catch (Exception e) {
					e.printStackTrace();
				}

			}
			for (Element div : div_results) {
			final	String asin = div.attr("name");
				
				if (!ASINKeyword.checkDuplicate(connection, asin)) {
				try{	
				final Document	document = getConnection(
							div.select("div.image a").attr("href")).get();
					
					String productName = document.select("span#btAsinTitle")
							.text();
					String by = document
							.select("div.buying:has(h1.parseasinTitle)")
							.select("span:contains(by) a").text();
					String productDescription = document.select(
							"div.productDescriptionWrapper").text();

					String price_str = document.select("span#actualPriceValue")
							.text();
					double price = 0;
					if (price_str.length() > 0)
						try {
							price = Double.parseDouble(price_str.substring(1));
						} catch (Exception e) {
							// TODO: handle exception
						}
					String ship_str = document.select(
							"span#pricePlusShippingQty span.plusShippingText")
							.text();
					double ship = 0;
					if (ship_str.contains("shipping")) {
						ship_str = ship_str.split("shipping")[0].trim();
						ship_str = ship_str.trim();
						try {
							ship = Double.parseDouble(ship_str.substring(4)
									.replaceFirst("^[\\x00-\\x200\\xA0]+", "")
									.replaceFirst("[\\x00-\\x20\\xA0]+$", ""));

						} catch (Exception e) {
							// e.printStackTrace();// TODO: handle exception
						}

					}
					String buyingChoices_str = document.select(
							"div#secondaryUsedAndNew div.mbcOlpLink a").text();
					int buyingChoices = 0;

					if (buyingChoices_str.length() > 0) {
						try {
							buyingChoices = Integer
									.parseInt(getNumber(buyingChoices_str));

						} catch (Exception e) {
							// TODO: handle exception
						}

					}
					//
					executor.execute(new Thread(){public void run() {downloadImage(getURLImages(document),asin);};})  ;
					ASINKeyword asinKeyword = new ASINKeyword();
				
					asinKeyword.setASIN(asin);
				    asinKeyword.setProductName(productName);
					asinKeyword.setBuyingChoices(buyingChoices);
					asinKeyword.setBy(by);
					asinKeyword.setPrice(price);
					asinKeyword.setProductDescription(productDescription);
					asinKeyword.setShipPrice(ship);
					asinKeyword.setItem_no(searchAsin.getItem_no());
					List<ASINProductDetails> list_pr=getProductDetails(document, asin);
					List<ASINTechnicalDetails> list_dt=getTechnicalDetails(document, asin);
					List<ASINReviews> list_rv= getAsinReviews(document, asin);
					ASINKeyword.insert(connection, asinKeyword);
					ASINItems.insert(connection, asin, searchAsin.getItem_no());
					ASINProductDetails.insert(connection, list_pr);
					ASINTechnicalDetails.insert(connection, list_dt);
					ASINReviews.insert(connection, list_rv);
					
				}catch(Exception ex){
					
					ex.printStackTrace();
				}}

			}

			page++;
			if (page > max_pages)
				loop = false;

		}

	}

	private Connection getConnection(String url) {

		return Jsoup
				.connect(url)
				.userAgent(
						"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:8.0) Gecko/20100101 Firefox/8.0")
				.header("Accept-Language", "en-us,en;q=0.5").timeout(100000);
	}

	private String getNumber(String text) {
		StringBuffer buffer = new StringBuffer();

		Pattern p = Pattern.compile("\\d+");
		Matcher m = p.matcher(text);
		while (m.find()) {
			buffer.append(m.group());
		}

		return buffer.toString();

	}

	private List<ASINProductDetails> getProductDetails(Document document,
			String asin) {
		List<ASINProductDetails> asinProductDetails = new ArrayList<ASINProductDetails>();

		Elements details = document.select(
				"td.bucket:contains(Product Details)").select(
				"div.content ul li");
		for (Element element : details) {
			ASINProductDetails productDetails = new ASINProductDetails();
			productDetails.setASIN(asin);
			productDetails.setDetailLine(element.text());
			asinProductDetails.add(productDetails);
		}

		return asinProductDetails;

	}

	private List<ASINTechnicalDetails> getTechnicalDetails(Document document,
			String asin) {
		List<ASINTechnicalDetails> list = new ArrayList<ASINTechnicalDetails>();
		Elements details = document.select(
				"div.bucket:contains(Technical Details)").select(
				"div.content ul li");
		for (Element element : details) {
			ASINTechnicalDetails technicalDetails = new ASINTechnicalDetails();
			technicalDetails.setASIN(asin);
			technicalDetails.setDetailLine(element.text());
			list.add(technicalDetails);
		}
		return list;

	}

	private List<ASINReviews> getAsinReviews(Document document1, String asin) {
		List<ASINReviews> list = new ArrayList<ASINReviews>();

		Elements span = document1.select("div#customerReviews div.content")
				.select("span.small:contains(customer reviews...)");
		if (span.size() > 0) {
			try {
				boolean loop =true;
				String url =span.select("a").attr("href");
				int page=0;
				while (loop) {
					
				
			Document	document = getConnection(url).get();
				Elements next = document.select("table.CMheadingBar span.paging").select("a:contains(Next)");
				page++;
				if( next.size()>0&&page<=10) url= next.attr("href"); else  loop=false;
				Elements reviews = document.select("table#productReviews")
						.select("div[style=margin-left:0.5em;]");
				for (Element element : reviews) {
					Elements div_title = element
							.select("div[style=margin-bottom:0.5em;]:has(span[class^=swSprite s_star_])");

					String start_str = div_title.select(
							"span[class^=swSprite s_star_]").attr("title");
					int stars = 0;
					if (start_str.contains("out of"))
						try {
							stars = (int) Double.parseDouble(start_str
									.split("out of")[0].trim());
						} catch (Exception e) {
							// TODO: handle exception
							e.printStackTrace();
						}
				
					String title = div_title.select(
							"span[style=vertical-align:middle;] b").text();
					String date_str = div_title.select(
							"span[style=vertical-align:middle;] nobr").text();
					SimpleDateFormat format = new SimpleDateFormat(
							"MMMM d, yyyy");
					Date reviewDat = new Date();
					try {
						reviewDat = format.parse(date_str);
					} catch (ParseException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
					Element div_author = div_title.first().nextElementSibling();

					String author = div_author.select(
							"span[style=font-weight: bold;]").text();
					// System.err.println(author);
					String location="";
					Elements  div_location = div_author
							.select("div[style=float:left;]:has(span[style=font-weight: bold;])");
					if (div_location.size()>0) location = div_location.first().ownText();
					if (location.contains("(") && location.contains(""))
						location = location.substring(
								location.indexOf("(") + 1,
								location.lastIndexOf(")"));
					else
						location = "";
					String reviewText = element.ownText();
					String helpful ="";
					ASINReviews asinReviews = new ASINReviews();
					asinReviews.setASIN(asin);
					asinReviews.setAuthor(author);
					asinReviews.setHelpful(helpful);
					asinReviews.setLocation(location);
					asinReviews.setReviewDat(reviewDat);
					asinReviews.setReviewText(reviewText);
					asinReviews.setStars(stars);
					asinReviews.setTitle(title);
					list.add(asinReviews);

				}

			}} catch (IOException e) {
				// TODO Auto-generated catch block
				System.err.println("get customer reviews error");
				// e.printStackTrace();
			}

		}
		return list;
	}
	
	private List<String> getURLImages(Document document){
		List<String> list = new ArrayList<String>();
		Elements img_thum= document.select("div#PIAltImagesDiv td.productThumbnail td.tiny img");
		if(img_thum.size()>0){
			for (Element img : img_thum) {
				String url =img.attr("src");
				if(url.startsWith("http")&&!url.contains("no-image-avail-img"))
				list.add(url);
			}
			
		}else {
			Elements prodImage = document.select("img#prodImage");
			if(prodImage.size()>0) {
				String url =prodImage.first().attr("src");
				if(url.startsWith("http")&&!url.contains("no-image-avail-img"))
	list.add(url);}
			
		}
		return list;
	}
	private void  processDownload(String url, File file)  {
		try{
			
				
			org.jsoup.Connection connectionJsoup = Jsoup.connect(url).timeout(60000).ignoreContentType(true).userAgent("Mozilla/5.0 (Windows NT 5.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1");
			Response response= connectionJsoup.execute();
			byte[] rs= response.bodyAsBytes();
			
		
			FileOutputStream fos = new FileOutputStream(file);
			fos.write(rs);
		fos.close();
		//Thread.sleep(200);
		}catch (Exception e) {
		//	e.printStackTrace();
		}}
	private void downloadImage(List<String> urls,String asin){
		int i=0;
		if(urls.size()<1) return;
		for (String url : urls) {
			String ext=".jpg";
			try{
			 ext = url.substring(url.lastIndexOf("."));}catch (Exception e) {
				// TODO: handle exception
			}
			File dir = new File("images/"+asin+"/");
			File file;
			if (!dir.exists()) {
					dir.mkdir();
			
			}
			
			if(url.contains("SL75_AA30_")){
				if(i==0) file= new File("images/"+asin+"/"+asin+ext);else
					file = new File("images/"+asin+"/"+asin+"-"+i+ext); 
				processDownload(url, file);
				i++;
				file = new File("images/"+asin+"/"+asin+"-"+i+ext); 
				processDownload(url.replaceAll("SL75_AA30_", "SL500_AA300_"), file);
				i++;
				
			}else {
				if(i==0) file= new File("images/"+asin+"/"+asin+ext);else
					file = new File("images/"+asin+"/"+asin+"-"+i+ext); 
				processDownload(url, file);
				i++;
			}
		}
		
	}

}
